# -*- coding: utf8 -*-
import sys


from pihsdk import pihsdk

from goose3 import Goose
from goose3.text import StopWordsChinese

def main_handler(event, context):
    http_client = pihsdk('test','fass_page_crawelr','http://10.0.24.6:25000')
    tmp_item = http_client.send_get('/getpage')
    if tmp_item['status'] == '1':
        cur_task = tmp_item['data']
        print(cur_task)
        cur_url = cur_task['url']
        if cur_task['http_head'] == 0:
            cur_url = 'http://' + cur_url
        else:
            cur_url = 'https://' + cur_url
        
        print(cur_url)
        
        
        g = Goose({'browser_user_agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36','stopwords_class': StopWordsChinese})
        article = g.extract(url=cur_url)
        
    
        res_item = {
            "status":1,
            "device": "faas_page_crawler",
            "data":{
                "content": article.cleaned_text,
                "title": article.title,
                "keywords": article.infos['meta']['keywords'],
                "task_id": cur_task['id']
            }   
        }
        
        return http_client.send_post('/submitpage', res_item)